import tensorflow as tf

global_step = tf.Variable(0, trainable=False)

initial_learning_rate = .1
learn_rate = tf.train.exponential_decay(initial_learning_rate,
                                        global_step=global_step,
                                        decay_rate=0.9, decay_steps=10)
opt = tf.train.GradientDescentOptimizer(learn_rate)
add_global = global_step.assign_add(1)
with tf.Session() as sess:
    tf.global_variables_initializer().run()
    print(sess.run(learn_rate))
    for _ in range(20):
        g, rate = sess.run([add_global, learn_rate])
        print(g, rate)